sink("mturk.pilot.analysis-psrm-log.txt",append=F,type="output")

library(irr)

##
#Pilot Analysis
##
load('coding.pilot-psrm.RData')

#Exemplar Labels
exemplar <- subset(coding.pilot,coding.pilot$Input.label!='unknown')
mean(ifelse(exemplar$Input.label==exemplar$Answer.categories, 1, 0))


#Other Pilot Test Outcomes
coding.pilot <- subset(coding.pilot,coding.pilot$Input.label=='unknown')
coding.pilot$Input.label <- NULL
pilot.combined <- merge(coding.pilot,coding.pilot,by=c('Input.id'),all.x=TRUE)
pilot.combined <- unique(pilot.combined[,names(pilot.combined)])
pilot.combined <- pilot.combined[which(pilot.combined$WorkerId.x!=pilot.combined$WorkerId.y),]
pilot.combined <- pilot.combined[order(pilot.combined$Input.id),]
names(pilot.combined) <- c('doc.id','worker1.id','worker1.time','header','worker1.label','worker1.type','worker1.slant','worker1.top.level','worker1.lower.level','worker2.id','worker2.time','header2','worker2.label','worker2.type','worker2.slant','worker2.top.level','worker2.lower.level')

#Topic Agreement
pilot.combined$top.level.agreement <- ifelse(pilot.combined$worker1.top.level==pilot.combined$worker2.top.level, 1, 0)
mean(pilot.combined$top.level.agreement,na.rm=TRUE)

#Slant Assessment Correlation and Reliability
cor(pilot.combined$worker1.slant,pilot.combined$worker2.slant,use="pairwise.complete.obs")
kappa2(pilot.combined[,c('worker1.slant','worker2.slant')])

sink()